{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-12-14 21:15:57,069\tINFO worker.py:1489 -- Connecting to existing Ray cluster at address: 192.168.1.248:6379...\n",
      "2023-12-14 21:15:57,081\tINFO worker.py:1664 -- Connected to Ray cluster. View the dashboard at \u001b[1m\u001b[32m127.0.0.1:8265 \u001b[39m\u001b[22m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Failed to look up actor with name 'emb'. This could because 1. You are trying to look up a named actor you didn't create. 2. The named actor died. 3. You did not use a namespace matching the namespace of the actor.\n",
      "\u001b[36m(UDFWorker pid=534687, ip=192.168.1.243)\u001b[0m MODEL[emb] Init Model,It may take a while.\n",
      "\u001b[36m(UDFWorker pid=534687, ip=192.168.1.243)\u001b[0m MODEL[emb]  Local mode: Load model from local path (/home/byzerllm/models/bge-large-zh), consume the model server to prevent socket server leak.\n",
      "\u001b[36m(UDFWorker pid=534688, ip=192.168.1.243)\u001b[0m MODEL[emb] Init Model,It may take a while.\n",
      "\u001b[36m(UDFWorker pid=534688, ip=192.168.1.243)\u001b[0m MODEL[emb]  Local mode: Load model from local path (/home/byzerllm/models/bge-large-zh), consume the model server to prevent socket server leak.\n",
      "\u001b[36m(UDFWorker pid=534688, ip=192.168.1.243)\u001b[0m MODEL[emb] Successful to init model, time taken:3.4095799922943115s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[36m(UDFWorker pid=534687, ip=192.168.1.243)\u001b[0m MODEL[emb] Successful to init model, time taken:3.446164608001709s\n"
     ]
    }
   ],
   "source": [
    "code_search_path=[\"/home/byzerllm/softwares/byzer-retrieval-lib/\"]\n",
    "env_vars = {\"JAVA_HOME\": \"/home/byzerllm/softwares/jdk-21\",\n",
    "            \"PATH\":\"/home/byzerllm/softwares/jdk-21/bin:/home/byzerllm/.rvm/gems/ruby-3.2.2/bin:/home/byzerllm/.rvm/gems/ruby-3.2.2@global/bin:/home/byzerllm/.rvm/rubies/ruby-3.2.2/bin:/home/byzerllm/.rbenv/shims:/home/byzerllm/.rbenv/bin:/home/byzerllm/softwares/byzer-lang-all-in-one-linux-amd64-3.3.0-2.3.7/jdk8/bin:/usr/local/cuda/bin:/usr/local/cuda/bin:/home/byzerllm/.rbenv/shims:/home/byzerllm/.rbenv/bin:/home/byzerllm/miniconda3/envs/byzerllm-dev/bin:/home/byzerllm/miniconda3/condabin:/home/byzerllm/.local/bin:/home/byzerllm/bin:/usr/local/cuda/bin:/usr/local/bin:/usr/bin:/usr/local/sbin:/usr/sbin:/home/byzerllm/.rvm/bin:/home/byzerllm/.rvm/bin\"}\n",
    "\n",
    "import os\n",
    "os.environ[\"RAY_DEDUP_LOGS\"] = \"0\" \n",
    "\n",
    "from byzerllm.utils.client import Templates\n",
    "\n",
    "import ray\n",
    "from byzerllm.utils.retrieval import ByzerRetrieval\n",
    "from byzerllm.utils.client import ByzerLLM,LLMRequest,LLMResponse,LLMHistoryItem,InferBackend\n",
    "from byzerllm.records import SearchQuery\n",
    "\n",
    "ray.init(address=\"auto\",namespace=\"default\",ignore_reinit_error=True,\n",
    "                 job_config=ray.job_config.JobConfig(code_search_path=code_search_path,\n",
    "                                                      runtime_env={\"env_vars\": env_vars})\n",
    "                 )   \n",
    "\n",
    "# ray.init(address=\"auto\",namespace=\"default\")   \n",
    "\n",
    "retrieval = ByzerRetrieval()\n",
    "retrieval.launch_gateway()\n",
    "\n",
    "llm = ByzerLLM()\n",
    "chat_model_name = \"chat\"\n",
    "\n",
    "# llm.undeploy(chat_model_name)\n",
    "# llm.undeploy(\"emb\")\n",
    "\n",
    "\n",
    "# model_type = \"deepseek\"\n",
    "# model_type = \"openbuddy\"\n",
    "# model_type = \"Qwen\"\n",
    "model_type = \"zephyr\"\n",
    "\n",
    "if model_type == \"deepseek\":\n",
    "    model_location=\"/home/byzerllm/models/deepseek2-llm-67b-chat\"\n",
    "    max_model_len = 4096\n",
    "    gpus_per_worker = 8\n",
    "\n",
    "if model_type == \"llama2\":\n",
    "    model_location=\"/home/byzerllm/models/openbuddy-llama2-70b-v10.1-bf16\"\n",
    "    max_model_len = 4000\n",
    "    gpus_per_worker = 8\n",
    "\n",
    "if model_type == \"Qwen\":\n",
    "    llm.setup_template(\"chat\",Templates.qwen())\n",
    "    model_location=\"/home/byzerllm/models/Qwen-72B-Chat\"\n",
    "    max_model_len = 24000\n",
    "    gpus_per_worker = 8\n",
    "\n",
    "if model_type == \"zephyr\": \n",
    "    model_location=\"/home/byzerllm/models/openbuddy-zephyr-7b-v14.1\"   \n",
    "    max_model_len = 24000\n",
    "    gpus_per_worker = 4\n",
    "\n",
    "llm.setup_max_model_length(\"chat\",max_model_len)\n",
    "\n",
    "if not llm.is_model_exist(\"chat\"):\n",
    "    llm.setup_gpus_per_worker(gpus_per_worker).setup_num_workers(1).setup_infer_backend(InferBackend.VLLM)\n",
    "    llm.deploy(\n",
    "        model_path=model_location,\n",
    "        pretrained_model_type=\"custom/auto\",\n",
    "        udf_name=chat_model_name,\n",
    "        infer_params={\"backend.max_num_batched_tokens\":24000,\n",
    "                      \"backend.max_model_len\":max_model_len}\n",
    "    )\n",
    "\n",
    "\n",
    "if not llm.is_model_exist(\"emb\"):\n",
    "    llm.setup_gpus_per_worker(0.4).setup_num_workers(2).setup_infer_backend(InferBackend.Transformers)\n",
    "    llm.deploy(\n",
    "        model_path=\"/home/byzerllm/models/bge-large-zh\",\n",
    "        pretrained_model_type=\"custom/bge\",\n",
    "        udf_name=\"emb\",\n",
    "        infer_params={}\n",
    "    )    \n",
    "\n",
    "llm.setup_default_model_name(chat_model_name) \n",
    "llm.setup_default_emb_model_name(\"emb\") \n",
    "\n",
    "# ray.kill(ray.get_actor(\"data_analysis\"))\n",
    "# 如果存储集群不存在，启动一个存储集群\n",
    "if not retrieval.is_cluster_exists(\"data_analysis\"):\n",
    "    builder = retrieval.cluster_builder()\n",
    "    builder.set_name(\"data_analysis\").set_location(\"/tmp/data_analysis\").set_num_nodes(2).set_node_cpu(1).set_node_memory(\"3g\")\n",
    "    builder.set_java_home(env_vars[\"JAVA_HOME\"]).set_path(env_vars[\"PATH\"]).set_enable_zgc()\n",
    "    builder.start_cluster()    \n",
    "\n",
    "def show_code(lang,code_string):\n",
    "    from IPython.display import display, Markdown    \n",
    "    display(Markdown(\"```{}\\n{}\\n```\".format(lang,code_string)))\n",
    "\n",
    "\n",
    "def show_text(msg):\n",
    "    from IPython.display import display, Markdown\n",
    "    display(Markdown(\"```{}\\n{}\\n```\".format(\"text\",msg))) \n",
    "\n",
    "def show_image(content):\n",
    "    from IPython.display import display, Image\n",
    "    import base64             \n",
    "    img = Image(base64.b64decode(content))\n",
    "    display(img)    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[LLMResponse(output=[0.003487164154648781, -0.009687656536698341, -0.026311276480555534, -0.004805233795195818, 0.03333209827542305, 0.014389371499419212, 0.017725428566336632, -0.008188240230083466, 0.008683566004037857, 0.013795615173876286, -0.004662488587200642, -0.01721588335931301, 0.007674216292798519, -0.036708295345306396, -0.010143325664103031, 0.03057188168168068, 0.004054826684296131, 0.007601254619657993, 0.021168725565075874, -0.043669793754816055, -0.004535554442554712, -0.034899331629276276, -0.02656671404838562, -0.007852491922676563, -0.011296529322862625, 0.0011517584789544344, 0.003761491971090436, 0.036769669502973557, 0.005315430928021669, -0.0304831825196743, -0.03222383186221123, 0.03505197912454605, 0.016987627372145653, -0.0006790258339606225, 0.009743181988596916, 0.022959737107157707, -0.031860850751399994, 0.007448054384440184, -0.02996632643043995, -0.0014919526875019073, -0.013263804838061333, -0.007554023526608944, -0.01376862172037363, -0.034408316016197205, 0.018597548827528954, -0.0072209183126688, -0.027406541630625725, -0.011339222081005573, -0.020509768277406693, -0.009664623998105526, 0.0330386646091938, -0.006910271011292934, 0.013683156110346317, 0.031918250024318695, -0.0441196970641613, 0.036308735609054565, 0.010654173791408539, -0.04150593653321266, 0.011574207805097103, -0.03883562237024307, 0.006603465881198645, 0.17114593088626862, -0.02017168700695038, -0.010195241309702396, -0.000662007019855082, 0.0324428528547287, 0.013261805288493633, -0.036180686205625534, 0.01150499377399683, -0.006519390270113945, -0.004848468117415905, -0.017690105363726616, 0.0006613082950934768, 0.011129829101264477, 0.014227084815502167, 0.005412760190665722, -0.003477417165413499, 0.010960256680846214, 0.030958393588662148, 0.011563673615455627, -0.0014258075971156359, -0.008992396295070648, 0.0197122935205698, -0.010717540979385376, 0.03347568213939667, -0.014771472662687302, 0.026119044050574303, 0.6796592473983765, -0.03673512488603592, -0.009683356620371342, 0.013677976094186306, 0.006932745687663555, -0.021039269864559174, 0.016656849533319473, 0.014686436392366886, 0.006964570377022028, -0.000702394638210535, -0.0037960412446409464, -0.00881967507302761, 0.004468253813683987, 0.021274618804454803, -0.030099209398031235, -0.024472413584589958, 0.015028843656182289, -0.026518650352954865, 0.0031476416625082493, -0.021398331969976425, -0.042368289083242416, 0.008516741916537285, -0.003708734642714262, -0.00909481942653656, -0.015467310324311256, -0.011309328489005566, -0.009143365547060966, -0.012118346057832241, 0.0009249071590602398, 0.0032388344407081604, -0.0003274680348113179, 0.018032854422926903, -0.00741921691223979, -0.012998362071812153, 0.010867539793252945, 0.011595121584832668, -0.03515337407588959, 0.002084673149511218, -0.00404240982607007, 0.048867568373680115, 0.014629336073994637, 0.03691813349723816, 0.035860542207956314, -0.014592318795621395, 0.004479689989238977, -0.014250054024159908, -0.017803611233830452, -0.004467634484171867, 0.012042558752000332, -0.005335631780326366, -0.027443354949355125, -0.0005584975588135421, -0.000650092784781009, 0.025340920314192772, -0.01659531518816948, 0.023712394759058952, -0.023007571697235107, 0.0406491756439209, -0.030652888119220734, -0.033632807433605194, 0.017339827492833138, 0.04892687872052193, -0.047924358397722244, 0.041657812893390656, -0.007443091366440058, -0.057748645544052124, -0.011978920549154282, -0.02966626174747944, 0.002320491475984454, 0.027939576655626297, -0.007535999175161123, -0.016538871452212334, -0.0071924529038369656, 0.020517168566584587, -0.0072575840167701244, 0.0052316621877253056, -0.028796061873435974, -0.011635727249085903, -0.013494555838406086, 0.010666527785360813, 0.0002971715584862977, -0.019710777327418327, 0.03233019635081291, -0.030934609472751617, 0.01860910654067993, -0.00542945321649313, 0.02042807638645172, 0.009309951215982437, 0.025962520390748978, -0.003915196284651756, 0.04057596996426582, 0.01234076265245676, -0.0008757482864893973, 0.024085985496640205, -0.008505471982061863, 0.016658678650856018, -0.008786945603787899, -0.03596913442015648, 0.0007207594462670386, 0.01668531633913517, -0.006616797298192978, 0.014478928409516811, 0.02174672856926918, 0.0015613242285326123, 0.01445790659636259, 0.009462420828640461, -0.00515639828518033, 0.008152217604219913, -0.03592684864997864, -0.014523912221193314, -0.013357413932681084, 0.012347275391221046, -0.010361616499722004, -0.007132135797291994, -0.0029083117842674255, 0.041163258254528046, -0.004088228568434715, -0.012642704881727695, -0.0055987657979130745, 0.027468694373965263, -0.022684529423713684, 0.005965694785118103, 0.0007612057379446924, -0.023586304858326912, 0.010202751494944096, 0.031223636120557785, -0.012289216741919518, -0.02580418810248375, -0.0029186708852648735, -0.0026317269075661898, 0.016747640445828438, 0.019786138087511063, -0.0007009890396147966, 0.014967741444706917, -0.02064170315861702, 0.01338962186127901, -0.028340231627225876, 0.02232782542705536, 0.017966287210583687, 0.004491711966693401, -0.0016070770798251033, 0.005132701247930527, -0.028016678988933563, -0.03519721329212189, -0.004375019110739231, 0.004931834060698748, -0.026293016970157623, -0.022160720080137253, -0.021147703751921654, -0.008610487915575504, 0.011130276136100292, -0.07148369401693344, -0.00416305148974061, -0.000805175572168082, -0.002656482858583331, -0.021040773019194603, -0.04544305428862572, 0.01453280821442604, 0.00682342192158103, -0.02436007559299469, 0.018760843202471733, 0.025740956887602806, 0.008858508430421352, -0.01149742305278778, -0.0247403085231781, -0.026157358661293983, -0.07064972817897797, 0.024480178952217102, -0.041041381657123566, -0.012050554156303406, -0.024317346513271332, 0.015078969299793243, 0.04207950457930565, -0.02701546624302864, -0.0336633138358593, 0.02473188005387783, -0.0034996250178664923, -0.009324630722403526, 0.021314484998583794, -0.0013195822248235345, 0.02465699426829815, -0.03275822475552559, 0.0070115067064762115, 0.0016848486848175526, -0.020131845027208328, -0.030705129727721214, -0.01643321104347706, 0.003340697381645441, 0.039595216512680054, -0.048301711678504944, 0.0066279033198952675, 0.021883903071284294, -0.023761160671710968, 0.06028076261281967, -0.02300170436501503, -0.025108633562922478, -0.004084618296474218, -0.0015410343185067177, -0.00048529644845984876, -0.03783874958753586, 0.008330194279551506, 0.014394514262676239, 0.015049230307340622, -0.01033846940845251, 0.022922445088624954, 0.019050363451242447, -0.039095938205718994, -0.0007338922005146742, 0.022936616092920303, 0.033093661069869995, -0.033233776688575745, -0.019860027357935905, -0.038662660866975784, -0.009848609566688538, -0.020621299743652344, 0.0049803960137069225, 0.025266768410801888, 0.010610099881887436, 0.02380465716123581, -0.02576163038611412, 0.0005014448543079197, -0.001553943264298141, 0.02610088139772415, 0.0061146714724600315, -0.03160164877772331, 0.02750554494559765, -0.0005143636954016984, 0.027386518195271492, -0.006704701576381922, 0.03896261379122734, 0.002452149987220764, -0.010115906596183777, 0.03585993871092796, 0.0042442865669727325, 0.016008326783776283, -0.025066222995519638, -0.03984496742486954, 0.028144018724560738, -0.002400975441560149, -0.003985360264778137, 0.016985299065709114, -0.0005925213918089867, -0.003142156871035695, 0.02630738727748394, 0.021557435393333435, -0.041893020272254944, 0.02467920631170273, -0.022686608135700226, 0.0021290306467562914, 0.023899594321846962, 0.0020760393235832453, -0.03380101919174194, 0.025044647976756096, 0.01835312508046627, 0.046550165861845016, 0.009988140314817429, -0.022746054455637932, -0.004761714953929186, -0.02097194641828537, -0.0012335729552432895, -0.03015459142625332, 0.021233169361948967, 0.00766024412587285, 0.0041959211230278015, 0.040278881788253784, 0.020252792164683342, -0.015014105476439, 0.004885601811110973, -0.017425110563635826, -0.001719424850307405, -0.0048691523261368275, 0.0005406413110904396, 0.023354997858405113, -0.01226485799998045, 0.009801914915442467, -0.0038793564308434725, 0.01994386501610279, -0.025081075727939606, -0.003334210952743888, -0.033486757427453995, 0.0005723822978325188, -0.05995246022939682, 0.007975487969815731, 0.025314727798104286, 0.000668104097712785, 0.016089992597699165, -0.04431256651878357, 0.01583111099898815, -0.02815023995935917, -0.004461910575628281, 0.016702432185411453, -0.02732214890420437, 0.008623084984719753, -0.027477486059069633, 0.010453798808157444, 0.03260448947548866, -0.0026187747716903687, 0.02661612629890442, -0.007653359789401293, 0.032506901770830154, 0.013677278533577919, 0.022244414314627647, 0.012560770846903324, -0.022455357015132904, -0.02440635673701763, 0.01599557138979435, -0.028186017647385597, -0.011854746378958225, 0.0013135905610397458, -0.009951449930667877, -0.008932115510106087, -0.03078976646065712, 0.027642086148262024, -0.002188103972002864, -0.03304843232035637, 0.025286225602030754, -0.2091224193572998, 0.027270575985312462, -0.03535797446966171, -0.009781043976545334, -0.019557591527700424, 0.032985132187604904, -0.0020474595949053764, -0.023344777524471283, -0.0426286980509758, -0.012007934972643852, -0.03006923198699951, 0.0007082742522470653, -0.020964475348591805, 0.021713698282837868, 0.011858331970870495, -0.0028324162121862173, -0.02072029747068882, 0.052678294479846954, -0.026862138882279396, 0.0208378117531538, 0.02712433785200119, 0.006862398702651262, 0.007543110754340887, 0.022442767396569252, 0.037888698279857635, 0.007933497428894043, 0.003239267971366644, 0.018849477171897888, -2.5156632545986213e-05, -0.017728624865412712, 0.02303941920399666, 0.007537879049777985, 0.022821828722953796, -0.006924267392605543, -0.006510318256914616, 0.03094233199954033, 0.015414939261972904, 0.02311880886554718, -0.016332542523741722, -0.02819601446390152, -0.008539386093616486, 0.008599283173680305, -0.00035821579513140023, -0.005728539079427719, -0.04077481850981712, -0.0374082587659359, -0.008709946647286415, 0.014195548370480537, -0.02397126890718937, 0.03182493522763252, 0.02329140156507492, -0.0005057370872236788, -0.013419192284345627, -0.0022248360328376293, -0.0019482006318867207, -0.008630066178739071, 0.0008408388821408153, -0.00757185323163867, -0.004615491256117821, 0.01832573674619198, -0.007680994458496571, -0.02333017997443676, 0.015432213433086872, -0.009587675333023071, 0.022644266486167908, 0.00421725120395422, 0.025885742157697678, -0.02249753847718239, 0.007867870852351189, 0.0015404230216518044, -0.02654317207634449, 0.002080630511045456, 0.02641434408724308, 0.02655085176229477, 0.028145700693130493, -0.021776242181658745, 0.023981569334864616, 0.011771087534725666, -0.003392221638932824, -0.035160474479198456, 0.028622962534427643, -0.0004344264161773026, 0.014902891591191292, -0.002243954688310623, 0.0013095729518681765, -0.04524563252925873, 0.004456636030226946, -0.004449646454304457, -0.00826727319508791, 0.01495937630534172, 0.019716791808605194, -0.0395483635365963, 0.001187824411317706, -0.01051784586161375, 0.0023222605232149363, 0.02146873064339161, 0.0045225429348647594, 0.01389208622276783, -0.00550555856898427, -0.006299915723502636, 0.014651183038949966, -0.008735722862184048, 0.008793179877102375, -0.0015218026237562299, -0.0028998625930398703, -0.025110550224781036, 0.03239241987466812, -0.023645328357815742, -0.016779035329818726, -0.008243058808147907, 0.022374486550688744, -0.0041848099790513515, -0.009755874052643776, 0.0699262022972107, 0.006078628357499838, 0.03077680617570877, 0.019532181322574615, 0.019421612843871117, 0.00958671048283577, 0.03784814476966858, 0.019062835723161697, 0.02165999449789524, -0.04340424761176109, 0.009812338277697563, -0.004814330022782087, -0.0011423665564507246, -0.004818270448595285, -0.03462719917297363, -0.02745284140110016, -0.05128570273518562, 0.0048982840962708, -0.002057109260931611, 0.012723682448267937, -0.004870827775448561, 0.0021779348608106375, -0.0011722864583134651, 0.010608060285449028, -0.004929748363792896, 0.010195843875408173, 0.000712362234480679, -0.018579507246613503, 0.006589855067431927, -0.027700915932655334, -0.020488061010837555, 0.016283748671412468, 0.010732755996286869, 0.012920831330120564, -0.00937565416097641, 0.010217326693236828, 0.017267189919948578, -0.05158461257815361, -0.025182759389281273, 0.005485102068632841, -0.02903989888727665, -0.03321501985192299, 0.013596141710877419, 0.010326729156076908, -0.004942576866596937, 0.002241370966657996, 0.0013867142843082547, -0.004122150596231222, 0.008285152725875378, 0.007885832339525223, -0.042544323951005936, -0.005666619632393122, -0.0009580635814927518, 0.012412006966769695, 0.006386008113622665, -0.004307236056774855, -0.07244092226028442, 0.027071408927440643, 0.010087007656693459, -0.029685815796256065, -0.004104976076632738, 0.008258511312305927, -0.02007545530796051, 0.0005800002254545689, -0.0029284143820405006, 0.0009169483673758805, -0.005889962427318096, -0.008580750785768032, 0.016074344515800476, 9.80365221039392e-05, -0.002231647027656436, -0.010334480553865433, -0.015868421643972397, 0.016856057569384575, 0.005081677343696356, 0.002056341851130128, -0.04106210544705391, 0.015522105619311333, 0.0047235144302248955, -0.00966882798820734, 0.0047174557112157345, 0.0049748653545975685, 0.01667879894375801, -0.008821837604045868, -0.021471302956342697, 0.002877107122913003, -0.006226821336895227, -0.0203151423484087, -0.021684791892766953, 0.010743357241153717, 0.00913991779088974, 0.004740656819194555, -0.007322159595787525, 0.016730116680264473, -0.018723344430327415, -0.009738635271787643, -0.000291364238364622, 0.006282579619437456, 0.003145459806546569, -0.02776404097676277, -0.018273627385497093, 0.04322375729680061, -0.01053323782980442, -0.02860885299742222, 0.01276377122849226, 0.04144861176609993, -0.015284676104784012, -0.026345200836658478, -0.028101766481995583, 0.015193703584372997, -0.03196451812982559, 0.014859961345791817, 0.01481930073350668, -0.018974918872117996, 0.009569328278303146, 0.03562072664499283, -0.04404635727405548, -0.0056460206396877766, 0.026976536959409714, -0.026551127433776855, -0.0031852456741034985, 0.024918364360928535, 0.02589736506342888, -0.005574916489422321, -0.003448277013376355, -0.018013782799243927, -0.003824896877631545, -0.008955924771726131, -0.006724239327013493, -0.028318079188466072, 0.0074127051047980785, -0.016823988407850266, -0.022451670840382576, -0.015015190467238426, -0.01244905311614275, -0.02381324954330921, -0.0077910958789289, 0.0013481685891747475, -0.014487778767943382, 0.01595621183514595, -0.0034461221657693386, -0.015891429036855698, 0.020495804026722908, 0.012493071146309376, -0.005790275521576405, -0.006006727460771799, -0.004146607127040625, 0.007108352612704039, -0.014651152305305004, -0.04931686446070671, 0.017160987481474876, 0.04614429548382759, 0.004196841735392809, -0.0002881951513700187, -0.012310288846492767, -0.004288583528250456, -0.012283885851502419, 0.012658577412366867, 0.017172737047076225, -0.009416626766324043, 0.008621850982308388, -0.01105703879147768, 0.004364383406937122, 0.004409303423017263, 0.017574066296219826, -0.07420620322227478, 0.012963857501745224, -0.02328185737133026, 0.008682440966367722, -0.03857184946537018, -0.02588002011179924, 0.03541257232427597, 0.010199875570833683, 0.003953305538743734, -0.0456959530711174, -0.005610710475593805, 0.012816651724278927, 0.020729098469018936, 0.001479317550547421, 0.01728593371808529, 0.02349201589822769, -0.036373525857925415, 0.006792224477976561, 0.047496676445007324, -0.04227810725569725, 0.044918011873960495, -0.027313312515616417, 0.001485234359279275, 0.04195864126086235, 0.01850823499262333, 0.014458553865551949, 0.0007959147915244102, -0.0035793643910437822, -0.014250915497541428, -0.042456742376089096, -0.03909331187605858, 0.0018750972812995315, -0.03276743367314339, -0.007875500246882439, -0.01903262548148632, -0.020034924149513245, 0.008563929237425327, -0.013778594322502613, 0.0040899342857301235, -0.015156956389546394, 0.017067372798919678, -0.01800251565873623, -0.006237392779439688, -0.004381848499178886, -0.03059084713459015, 0.015515659004449844, 0.014665579423308372, 0.010791747830808163, -0.0041982620023190975, 0.036634720861911774, -7.476412429241464e-05, -0.011273895390331745, -0.009780917316675186, 0.015465553849935532, -0.010003117844462395, -0.011319766752421856, -0.05029883608222008, -0.027274155989289284, -0.012175329960882664, -0.011207429692149162, 0.038728877902030945, 0.018526911735534668, 0.028016095981001854, -0.05741608887910843, -0.0441848449409008, 0.0007920467178337276, 0.0029930616728961468, 0.015235758386552334, -0.014595159329473972, -0.028311071917414665, 0.003402797970920801, -0.013912631198763847, -0.025469908490777016, 0.001964088762179017, 0.022597473114728928, -0.0026342759374529123, -0.004093795083463192, -0.012308330275118351, 0.007577384822070599, -0.004834438674151897, -0.01316345389932394, -0.00784927885979414, -0.02260562963783741, -0.020643990486860275, -0.033291444182395935, -0.027470029890537262, -0.010541259311139584, -0.006558713503181934, 0.018972892314195633, -0.0183301642537117, 0.024640390649437904, -0.02136806771159172, -0.006239387672394514, -0.010789881460368633, -0.025372672826051712, 0.024040713906288147, 0.015895482152700424, -0.010213576257228851, -0.024381348863244057, -0.0448463000357151, -0.004667638335376978, 0.02452213689684868, -0.051004838198423386, -0.04308260977268219, 0.0028385701589286327, 0.03381859138607979, 0.005350567400455475, -0.00975355040282011, -0.022160837426781654, 0.020557278767228127, 0.011655430309474468, 0.013543829321861267, -0.03770613297820091, -0.020391646772623062, 0.00497646676376462, -0.02548833191394806, 0.012297675013542175, -0.017847741022706032, -0.0067238896153867245, -0.04312867298722267, 0.007704803720116615, -0.021088413894176483, 0.0039205471985042095, -0.021554043516516685, -0.020493799820542336, 0.0011133376974612474, -0.034021273255348206, -0.034975796937942505, 0.013115854933857918, -0.01868431642651558, -0.03380630165338516, 0.018172260373830795, -0.008293910883367062, -0.00013101330841891468, 0.031108330935239792, 0.018828511238098145, -0.028270458802580833, 0.014735528267920017, 0.002914264565333724, 0.029054060578346252, 0.00045350147411227226, -0.0211033895611763, 0.010446160100400448, 0.023129625245928764, 0.01749914139509201, -0.008772444911301136, 0.035053323954343796, -0.03067728877067566, 0.004595721606165171, -0.016242122277617455, -0.003366845427080989, -0.002589859301224351, 0.025029564276337624, -0.0170635674148798, 0.018598606809973717, -0.016992313787341118, -0.032109666615724564, 0.009626663289964199, 0.001241601537913084, -0.013411203399300575, -0.022111525759100914, 0.018503334373235703, -0.026170773431658745, -0.014047595672309399, 0.0018095612758770585, 0.0007818780140951276, 0.029101718217134476, -0.010108111426234245, 0.0003786681336350739, -0.013199646025896072, -0.0006521902978420258, 0.038488637655973434, 0.032452281564474106, -0.013090711086988449, 0.04097181186079979, 0.020671872422099113, -0.012920120730996132, 0.00041757512371987104, -0.01123118493705988, -0.023483628407120705, -0.020343493670225143, 0.0013833548873662949, 0.010726730339229107, -0.005514190997928381, -0.00679475674405694, -0.02707125060260296, 0.002245566574856639, 0.023366492241621017, 0.032739054411649704, 0.028669873252511024, 0.006854634266346693, -0.008089341223239899, 0.012980143539607525, -0.00509857852011919, -0.0022831056267023087, -0.0256450567394495, 0.0035622180439531803, -0.0035301006864756346, 0.01533513143658638, -0.014974132180213928, 0.03006555326282978, -0.025818681344389915, -0.02253497950732708, 0.012122238986194134, -0.024009786546230316, -0.013579520396888256, -0.014208106324076653, 0.03109326958656311, 0.0002320162020623684, -0.00023528176825493574, -0.014989085495471954, -0.004432328045368195, 0.03095516562461853, 0.009784413501620293, -0.006058461498469114, -0.030436335131525993, -0.01728566363453865, -0.01724725216627121, -0.009757598862051964, 0.006680123042315245, 0.011081486940383911, 0.014647094532847404, 0.01161315105855465, 0.017406942322850227, 0.0032805914524942636, -0.02726162225008011, 0.006285037379711866, 0.00023638691345695406, 0.008495407178997993, -0.012316248379647732, -0.020926538854837418, -0.015125715173780918, 0.0020211702212691307, -0.006255616433918476, 0.023308975622057915, -0.0011762744979932904, -0.008038735948503017, -0.013104522600769997, -0.0033966589253395796, 0.002647778484970331, 0.02373378723859787, -0.008094641380012035, -0.015157176181674004, 0.007313108071684837, -0.001794176991097629, -0.013907947577536106, -0.020762789994478226, -0.0453525148332119, -0.039866719394922256, -0.030962055549025536, 0.02097724936902523, -0.03887280449271202, 0.011119849048554897, -0.011271895840764046, 0.03163275122642517, 0.029161088168621063, -0.008770296350121498, -0.01686720736324787, 0.00319055188447237, -0.002703374717384577, 0.0016899722395464778, -0.04344872385263443, 0.0026298724114894867, 0.00607090350240469, -0.01238968688994646, 0.012830028310418129, -0.022178171202540398, -0.024423353374004364, -0.006215061992406845, -0.014774901792407036, -0.05375397950410843, 0.017746206372976303, -0.01663937233388424, -0.007147969678044319, -0.035357411950826645, -0.0022933799773454666, 0.006639537867158651, 0.008462690748274326, -0.03339289873838425, 0.020321397110819817, 0.0007797889411449432, -2.2001282559358515e-05, 0.02070847898721695, -0.005172993056476116, -0.03954135999083519, 0.0014728072565048933, 0.005381183233112097, 0.025643538683652878, 0.004573381505906582, -0.014447255991399288, -0.016562940552830696, 0.0006292092148214579, -0.008723686449229717, -0.046139031648635864, -0.00773534644395113, -0.026355640962719917, -0.059140581637620926, 0.010546548292040825, -0.0022316824179142714, -0.03412408381700516, 0.009994631633162498, 0.06007905676960945, 0.03250957280397415, -0.007714328821748495, 0.004486148711293936, -0.003379211528226733, 0.013348080217838287, -0.012157591991126537, -0.014410416595637798, 0.02637338638305664, -0.03588853031396866, 0.04094064235687256, -0.00044753748807124794, 0.02942517399787903, 0.0002796211338136345, 0.021382110193371773, 0.013867533765733242, -0.03270706534385681, 0.012606589123606682, -0.024081112816929817, -0.00666817044839263, -0.02813187800347805, 0.008712918497622013, 0.0015491079539060593, -0.010899409651756287, 0.013588478788733482, 0.009669236838817596, -0.016476178541779518, 0.015617279335856438, -0.01465313509106636, 0.009878525510430336, 0.0073933107778429985, -0.01459471508860588, -0.011199007742106915, -0.019285207614302635, -0.03990950807929039, 0.019521983340382576, 0.0033835868816822767, -0.0011618291027843952, 0.003699801629409194, -0.009871391579508781, 0.009854156523942947, 0.027680665254592896], input={'instruction': '我试试看能不能把这句话转换成向量', 'embedding': True, 'max_length': 4096, 'top_p': 0.95, 'temperature': 0.1}, metadata={})]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "llm.emb(\"emb\",LLMRequest(instruction=\"我试试看能不能把这句话转换成向量\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from byzerllm.apps.agent import Agents\n",
    "\n",
    "from byzerllm.apps.agent.user_proxy_agent import UserProxyAgent\n",
    "from byzerllm.apps.agent.extensions.retrieval_agent import RetrievalAgent\n",
    "\n",
    "\n",
    "retrieval_agent = Agents.create_local_agent(RetrievalAgent,\"retrieval_agent\",llm,retrieval,\n",
    "                                   max_consecutive_auto_reply=1000,\n",
    "                                   code_agent = None\n",
    "                                   )\n",
    "\n",
    "user = Agents.create_local_agent(UserProxyAgent,\"user\",llm,retrieval,\n",
    "                                human_input_mode=\"NEVER\",\n",
    "                                max_consecutive_auto_reply=0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "user (to retrieval_agent):\n",
      "\n",
      "介绍下 Gluten\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "retrieval_agent (to user):\n",
      "\n",
      "Gluten是Apache Spark的一个本地化计算引擎，它结合了Spark的可扩展性框架、优化器、容错能力和Native执行库的高性能，以提高数据处理速度并降低计算成本。它可以与Velox或Clickhouse集成。BIGO大数据团队是参与该项目的研发团队之一，并已在生产环境中使用Gluten替代Spark的ETL工作负载，实现了成本节省。\n",
      "\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "user.initiate_chat(\n",
    "retrieval_agent,\n",
    "message={\n",
    "    \"content\":\"介绍下 Gluten\",\n",
    "    \"metadata\":{\n",
    "        \"file_path\":\"/home/byzerllm/projects/jupyter-workspace/aritcle.txt\"\n",
    "    }\n",
    "},)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "byzerllm-dev",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
